package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.main.ChengduLifeMain;
import com.spider.silence.main.YiBinZeroJuLiMain;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.YiBinZeroJuLiTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

/**
 * Created by qiuhao on 2017/3/28.
 */
public class YiBinZeroJuLiDownLoader extends AbstractCommonDownLoader implements Cloneable {
    private Logger logger = Logger.getLogger(YiBinZeroJuLiDownLoader.class);
    public static final String url = "http://bbs.ybvv.com/forum.php";
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }

    /**
     * 获得每个版块的url
     *
     * @return
     */
    public List<String> getModulesUrlList() {
//        List<String[]> list= ProxyPool.proxyList();
        List<String> urlList = new ArrayList<String>();
        Document document = null;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        boolean flag = true;
        while(flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.TimeOut"))) {
            try {
                Page pageContent = null;
//                String proxy=list.get(new Random().nextInt(list.size()))[2]+":"+list.get(new Random().nextInt(list.size()))[3];
                pageContent = getHtml(url, true);
                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 200) {
                    flag=false;
                    document = pageContent.getHtml().getDocument();
                    Element element = document.body();
                    Elements module = element.select(".bm").select(".bmw").select(".flg").select(".cl").select(".fl_g");
                    for (int i = 0; i < module.size(); i++) {
                        String moduleUrl = module.get(i).select("dt").select("a").attr("href");
                        urlList.add(moduleUrl);
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
                Logger logger = Logger.getLogger(YiBinZeroJuLiMain.class);
                logger.error(e.getMessage(),e);
            }finally {
                nowTime = System.currentTimeMillis();
            }
        }
        return urlList;
    }

    /**
     * 生成task 添加到TaskManager中（获取每个模块中第一页的帖子的url）
     *
     * @param urlList
     */
    public void generateTask(List<String> urlList) {
//        List<String[]> list= ProxyPool.proxyList();
//        if (urlList.size() > 0) {
            for (int i = 0; i < urlList.size(); i++) {
                boolean flag = true;
                Document document = null;
                long startTime = System.currentTimeMillis();
                long nowTime = startTime;
                while(flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.TimeOut"))) {
                    try {
//                        String proxy=list.get(new Random().nextInt(list.size()))[2]+":"+list.get(new Random().nextInt(list.size()))[3];
                        Page pageContent = getHtml(urlList.get(i), true);
                        if(pageContent==null){

                        }else if (pageContent.getStatusCode() == 200) {
                            flag=false;
                            document = pageContent.getHtml().getDocument();
                            Elements element = document.select("tbody[id^=normalthread]").select(".new");
                            for (int j = 0; j < element.size(); j++) {
                                String url = element.get(j).select(".s.xst").attr("href");
                                YiBinZeroJuLiTask task = new YiBinZeroJuLiTask(URLDecoder.decode(url, "UTF-8"));
                                CommonTaskManager.add("YiBinZeroJuLi", task);
                            }
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        Logger logger = Logger.getLogger(ChengduLifeMain.class);
                        logger.error(e.getMessage(),e);
                    }finally {
                        nowTime = System.currentTimeMillis();
                    }
                }
//            }
        }
    }


    /**
     * 获取历史帖子的url
     *
     * @param urlList
     */
    public void generateAllTask(List<String> urlList) {
//        List<String[]> list= ProxyPool.proxyList();
        for (int i = 0; i < urlList.size(); i++) {
            boolean flag = true;
            Document document = null;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while(flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.TimeOut"))) {
                try {
//                    String proxy=list.get(new Random().nextInt(list.size()))[2]+":"+list.get(new Random().nextInt(list.size()))[3];
                    Page pageContent = getHtml(urlList.get(i), true);
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 200) {
                        flag=false;
                        document = pageContent.getHtml().getDocument();
                        int page = 0;
                        if (document.select("#fd_page_top").select(".pg").size() == 0) {
                            page = 1;
                        } else {
                            page = Integer.parseInt(document.select("#fd_page_top").select(".pg").select("label").select("span").get(0).attr("title").split(" ")[1]); //页码
                        }
                        for (int j = 1; j < page + 1; j++) {
                            try {
                                 Document d = get(urlList.get(i).split("-")[0] + "-" + urlList.get(i).split("-")[1] + "-" + j + ".html", true);
                                Elements element = d.select(".s.xst");
                                if (element.size() > 0) {
                                    for (int k = 0; k < element.size(); k++) {
                                        String url = element.get(k).attr("href");
                                        YiBinZeroJuLiTask task = new YiBinZeroJuLiTask(URLDecoder.decode(url, "UTF-8"));
                                        CommonTaskManager.add("YiBinZeroJuLi", task);
                                    }
                                }
                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                                continue;
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    Logger logger = Logger.getLogger(ChengduLifeMain.class);
                    logger.error(e.getMessage(),e);
                }finally {
                    nowTime = System.currentTimeMillis();
                }
            }
        }
    }

    /**
     * @param url 帖子的url
     * @return 每个帖子的内容
     */
    public JSONObject crawlItem(String url) {
        files=new ConcurrentHashMap<>();
//        List<String[]> proxylist= ProxyPool.proxyList();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();
        int page = 1;
        Document document = null;
        try {
            boolean flag = true;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            if (!"".equals(url) && url != null) {
                while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.TimeOut"))) {
//                    String proxy=proxylist.get(new Random().nextInt(proxylist.size()))[2]+":"+proxylist.get(new Random().nextInt(proxylist.size()))[3];
                    Page pageContent = getHtml(url, true);
                    if (pageContent == null) {

                    } else if (pageContent.getStatusCode() == 200) {
                        try {
                            document = pageContent.getHtml().getDocument();
                            String title = document.select(".wp.cl").select(".ts").select("span").text(); //帖子标题
                            Element publishContent = document.select(".comiis_vrx").get(0);
                            Element head = document.select(".comiis_viewtop").get(0);
                            int read_count = Integer.parseInt(head.select(".hm.ptn").select("span").get(1).text()); //帖子查看次数
                            int reply_count = Integer.parseInt(head.select(".hm.ptn").select("span").get(4).text());//帖子回复次数
                            data.put("title", title);
                            data.put("read_num", read_count);
                            data.put("reply_count", reply_count);
                            data.put("url", url);
                            if(document.select(".pgs.mtm.mbm.cl").select(".pg").size()>0){
                                page = Integer.valueOf(document.select(".pgs.mtm.mbm.cl").select(".pg").select("label").select("span").attr("title").split(" ")[1]);//页数
                            }
                            Element publichTimeElement = publishContent.select(".comiis_vrx").get(0).select(".plc").select(".pti").select(".authi").get(0);
                            String publichTime = "";
                            if (publichTimeElement.select("em").select("span").hasAttr("title")) {
                                publichTime = publichTimeElement.select("em").select("span").attr("title");
                            } else {
                                publichTime = publichTimeElement.select("em").text().split(" ")[1] + " " + publichTimeElement.select("em").text().split(" ")[2];  //帖子发表时间
                            }
                            data.put("publish_time", publichTime);
                            Date now = new Date();
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                            data.put("spider_time", sdf.format(now)); //爬取时间
                            String content = publishContent.select(".t_fsz").select(".t_f").text();//剔除发布内容中的广
                            Elements imgEles=publishContent.select(".t_fsz").select(".t_f").select("img[zoomfile]");
                            JSONArray img_urls=new JSONArray();
                            for(int imgIndex=0;imgIndex<imgEles.size();imgIndex++){
                                Element imgEle=imgEles.get(imgIndex);
                                String img_url=imgEle.absUrl("zoomfile");
                                img_urls.add(img_url);
                            }
                            data.put("content", content); //发布内容
                            System.out.println(url);
                            int like_count = Integer.valueOf(publishContent.select(".mtw.mbm.hm.cl").get(0).select("a").get(6).select("span").text());//喜欢数
                            int dislike_count = Integer.valueOf(publishContent.select(".mtw.mbm.hm.cl").get(0).select("a").get(7).select("span").text());//不喜欢数
                            data.put("like_count", like_count);
                            data.put("dislike_count", dislike_count);
                            String publish_user = publishContent.select(".pls").get(0).select(".authi").select("a").text();//用户名
                            data.put("publish_user", publish_user);
                            String publish_user_photo = publishContent.select(".pls").get(0).select(".avatar").select("a").select("img").attr("src");
                            String publish_user_id = publishContent.select(".pls").get(0).select(".authi").select("a").attr("href").split("-")[2].split("\\.")[0];
                            data.put("publish_user_id", publish_user_id);
                            data.put("publish_user_photo", publish_user_photo);
                            String id = url.split("-")[1];//帖子id
                            data.put("id", id);
                            data.put("img_urls",img_urls);
                            long tsp = DateUtils.dateToTimestamp(DateUtils.parseDate(publichTime)).getTime();
                            String filePrefix = "YiBinZeroJuLi";//文件前缀
                            String fileName = filePrefix + "_" + tsp + "_" + publish_user_id + "_" + id;
                            String publishDate = DateUtils.formatDate(DateUtils.parseDate(publichTime)).replace("-", "-");
                            String speechesId = "YiBinZeroJuLi_speeches_" + publishDate + "_" + fileName;
                            File file = FileUtils.createTempFile(speechesId + "_pageNo_1", document.html().toString());
                            files.put(speechesId + "_pageNo_1", file);
                            //帖子首页的文件
                            //将本页面的回复添加上去
                            int size = document.select(".comiis_vrx").size();
                            Elements replyEntities = document.select(".comiis_vrx");
                            JSONArray reply_nodes = new JSONArray();
                            if(replyEntities.size()>2) { //有回复
                                for (int i = 1; i < replyEntities.size() - 1; i++) {
                                    try {
                                        JSONObject reply = new JSONObject();
                                        Element replyUserInfo = replyEntities.get(i).select(".pls").get(0);
                                        String replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                        String homeUrl = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                        String replyUserId = homeUrl.split("-")[2].split("\\.")[0];
                                        String replyUserPhoto = replyUserInfo.select(".avtm").select("img").attr("src");
                                        String replyContent = "";
                                        JSONArray reply_img_urls=new JSONArray();
                                        int reply_id = Integer.valueOf(replyEntities.get(i).attr("id").split("_")[1]);
                                        if (replyEntities.get(i).select(".t_fsz").size() > 0) {
                                            replyContent = replyEntities.get(i).select(".t_fsz").text();
                                            Elements replyImgEles=replyEntities.get(i).select(".t_fsz").select("img[zoomfile]");
                                            for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                                Element imgEle=replyImgEles.get(imgIndex);
                                                String img_url=imgEle.absUrl("zoomfile");
                                                reply_img_urls.add(img_url);
                                            }
                                        } else {
                                            replyContent = replyEntities.get(i).select(".pct").text();
                                            Elements replyImgEles=replyEntities.get(i).select(".pct").select("img[zoomfile]");
                                            for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                                Element imgEle=replyImgEles.get(imgIndex);
                                                String img_url=imgEle.absUrl("zoomfile");
                                                reply_img_urls.add(img_url);
                                            }
                                        }
                                        String replyTime = replyEntities.get(i).select(".authi").select("em").text().split(" ")[1] + " " + replyEntities.get(i).select(".authi").select("em").text().split(" ")[2];
                                        int reply_like_count = 0;
                                        int reply_dislike_count = 0;
                                        if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 3) {
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text().equals("")) {
                                                reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text());
                                            }
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text().equals("")) {
                                                reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text());
                                            }
                                        } else if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 2) {
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(0).select("span").text().equals("")) {
                                                reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(0).select("span").text());
                                            }
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text().equals("")) {
                                                reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text());
                                            }
                                        } else if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 4) {
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text().equals("")) {
                                                reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text());
                                            }
                                            if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(3).select("span").text().equals("")) {
                                                reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(3).select("span").text());
                                            }
                                        }
                                        String replyUrl = url + "#" + replyEntities.get(i).id();
                                        reply.put("publish_user", replyUser);
                                        reply.put("id", reply_id);
                                        reply.put("publish_user_id", replyUserId);
                                        reply.put("publish_user_photo", replyUserPhoto);
                                        reply.put("content", replyContent);
                                        reply.put("img_urls",reply_img_urls);
                                        reply.put("publish_time", replyTime);
                                        reply.put("url", replyUrl);
                                        reply.put("like_count", reply_like_count);
                                        reply.put("dislike_count", reply_dislike_count);
                                        reply.put("ancestor_id", id);
                                        if (replyEntities.get(i).select(".plc").select(".quote").size() > 0) { //回复的回复
                                            if (replyEntities.get(i).select(".plc").select(".quote").select("font").get(0).select("a").size() > 0) {
                                                String parentUrl = replyEntities.get(i).select(".plc").select(".quote").select("font").select("a").attr("href");
                                                String pid = "";
                                                if (parentUrl.contains("&amp;")) {
                                                    pid = parentUrl.split("&amp;")[2].split("=")[1];
                                                } else {
                                                    pid = parentUrl.split("&")[2].split("=")[1];
                                                }
                                                reply.put("parent_id", pid);
                                            }
                                        } else {//回复
                                            reply.put("parent_id", id);
                                        }
                                        reply_nodes.add(reply);
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                        logger.error(e.getMessage(),e);
                                        continue;
                                    }
                                }
                            }
                            JSONArray replys = new JSONArray();
                            List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                            if (page >= 2) {
                                int threadNum = Integer.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.ReplyThreadCount"));
                                List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                try {
                                    ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                    for (int i = 0; i < splitPages.size(); i++) {
                                        YiBinZeroJuLiDownLoader downLoader = this.cloneDownLoader();
                                        ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), url, id,speechesId);
                                        FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }
                                    int count = 0;
                                    for (FutureTask<JSONArray> futureTask : futureTasks) {
                                        JSONArray replyArrays = futureTask.get();
                                        for (int i = 0; i < replyArrays.size(); i++) {
                                            JSONArray array = (JSONArray) replyArrays.get(i);
                                            for(int j=0;j<array.size();j++){
                                               JSONObject object= array.getJSONObject(j);
                                                replys.add(object);
                                            }
                                        }
                                    }
                                    if (replys.size() > 0 && replys != null) {
                                        for (int i = 0; i < replys.size(); i++) {
                                            JSONObject reply_object = replys.getJSONObject(i);
                                            reply_nodes.add(reply_object);
                                        }
                                    }
                                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                                        for (int i = 0; i < reply_nodes.size(); i++) {
                                            JSONObject reply_object = reply_nodes.getJSONObject(i);
                                            Map<String, Object> map = reply_object;
                                            list.add(map);
                                        }
                                    }
                                    Collections.sort(list, new Comparator<Map<String, Object>>() {

                                        public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                            String s1 = o1.get("publish_time").toString();
                                            String s2 = o2.get("publish_time").toString();
                                            int flag = s1.compareTo(s2);
                                            return flag;
                                        }
                                    });
                                } catch (Exception e) {
                                    e.printStackTrace();
                                    logger.error(e.getMessage(),e);
                                } finally {
                                    executorService.shutdown();
                                }
                            } else {
                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                        JSONObject reply_object = reply_nodes.getJSONObject(i);
                                        Map<String, Object> map = reply_object;
                                        list.add(map);
                                    }
                                }

                            }
                            data.put("reply_nodes", list);

                        } catch (Exception e) {
                            e.printStackTrace();
                            logger.error(e.getMessage(), e);
                        } finally {
                            flag=false;
                            nowTime = System.currentTimeMillis();
                        }
                    }
                }
            }
        } catch (Exception e) {
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(), e);
            e.printStackTrace();
        }
        item.put("data", data);
        return item;
    }


    public class ReplyCrawler implements Callable<JSONArray> {
        private YiBinZeroJuLiDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
//            List<String[]> proxylist= ProxyPool.proxyList();
            JSONArray array = new JSONArray();
            for (int k = 0; k < pages.size(); k++) {
                JSONArray reply_nodes = new JSONArray();
                Document document = null;
                try {
                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while(flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("YiBinZeroJuLi.TimeOut"))) {
                        try{
                        String spliteUrl[] = url.split("-");
                        String getUrl = spliteUrl[0] + "-" + spliteUrl[1] + "-" + pages.get(k) + "-" + spliteUrl[3];
//                        String proxy = proxylist.get(new Random().nextInt(proxylist.size()))[2] + ":" + proxylist.get(new Random().nextInt(proxylist.size()))[3];
                        Page pageContent = downLoader.getHtml(getUrl, true);
                        if (pageContent == null) {

                        } else if (pageContent.getStatusCode() == 200) {
                            document = pageContent.getHtml().getDocument();
                            Elements replyEntities = document.select(".comiis_vrx");

                            for (int i = 0; i < replyEntities.size() - 1; i++) {
                                try {
                                    JSONObject reply = new JSONObject();
                                    Element replyUserInfo = replyEntities.get(i).select(".plhin").select(".pls").get(0);
                                    String replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                    String homeUrl = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                    String replyUserId = homeUrl.split("-")[2].split("\\.")[0];
                                    String replyUserPhoto = replyUserInfo.select(".avatar").select(".avtm").select("img").attr("src");
                                    String replyContent = "";
                                    JSONArray reply_img_urls=new JSONArray();
                                    int reply_id = Integer.valueOf(replyEntities.get(i).attr("id").split("_")[1]);
                                    if (replyEntities.get(i).select(".t_fsz").size() > 0) {
                                        replyContent = replyEntities.get(i).select(".t_fsz").text();
                                        Elements replyImgEles=replyEntities.get(i).select(".t_fsz").select("img[zoomfile]");
                                        for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                            Element imgEle=replyImgEles.get(imgIndex);
                                            String img_url=imgEle.absUrl("zoomfile");
                                            reply_img_urls.add(img_url);
                                        }
                                    } else {
                                        replyContent = replyEntities.get(i).select(".pct").html();
                                        Elements replyImgEles=replyEntities.get(i).select(".pct").select("img[zoomfile]");
                                        for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                            Element imgEle=replyImgEles.get(imgIndex);
                                            String img_url=imgEle.absUrl("zoomfile");
                                            reply_img_urls.add(img_url);
                                        }
                                    }
                                    String replyTime = replyEntities.get(i).select(".authi").select("em").text().split(" ")[1] + " " + replyEntities.get(i).select(".authi").select("em").text().split(" ")[2];
                                    int reply_like_count = 0;
                                    int reply_dislike_count = 0;
                                    if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 3) {
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text().equals("")) {
                                            reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text());
                                        }
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text().equals("")) {
                                            reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text());
                                        }
                                    } else if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 2) {
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(0).select("span").text().equals("")) {
                                            reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(0).select("span").text());
                                        }
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text().equals("")) {
                                            reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(1).select("span").text());
                                        }
                                    } else if (replyEntities.get(i).select(".pob.cl").select("em").select("a").size() == 4) {
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text().equals("")) {
                                            reply_like_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(2).select("span").text());
                                        }
                                        if (!replyEntities.get(i).select(".pob.cl").select("em").select("a").get(3).select("span").text().equals("")) {
                                            reply_dislike_count = Integer.parseInt(replyEntities.get(i).select(".pob.cl").select("em").select("a").get(3).select("span").text());
                                        }
                                    }

                                    String replyUrl = url + "#" + replyEntities.get(i).id();
                                    reply.put("publish_user", replyUser);
                                    reply.put("id", reply_id);
                                    reply.put("publish_user_id", replyUserId);
                                    reply.put("publish_user_photo", replyUserPhoto);
                                    reply.put("content", replyContent);
                                    reply.put("publish_time", replyTime);
                                    reply.put("url", replyUrl);
                                    reply.put("like_count", reply_like_count);
                                    reply.put("dislike_count", reply_dislike_count);
                                    reply.put("ancestor_id", parent_id);
                                    reply.put("img_urls",reply_img_urls);
                                    if (replyEntities.get(i).select(".quote").size() > 0) { //回复的回复
                                        if (replyEntities.get(i).select(".plc").select(".quote").select("font").get(0).select("a").size() > 0) {
                                            String parentUrl = replyEntities.get(i).select(".plc").select(".quote").select("font").select("a").attr("href");
                                            String pid = "";
                                            if (parentUrl.contains("&amp;")) {
                                                pid = parentUrl.split("&amp;")[2].split("=")[1];
                                            } else {
                                                pid = parentUrl.split("&")[2].split("=")[1];
                                            }
                                            reply.put("parent_id", pid);
                                        }
                                    } else {//回复
                                        reply.put("parent_id", parent_id);
                                    }
                                    reply_nodes.add(reply);
                                } catch (Exception e) {
                                    logger.error(e.getMessage(),e);
                                    e.printStackTrace();
                                    continue;//因为禁言无法获取的情况
                                 }
                                }
                            flag = false;
                            }
                        }catch (Exception e){
                            logger.error(e.getMessage(),e);
                        }finally {
                            if (flag == false) {
                                array.addAll(reply_nodes);
                            }
                            nowTime = System.currentTimeMillis();
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    continue;
                }
                array.add(reply_nodes);
            }
            return array;
        }

        public ReplyCrawler(YiBinZeroJuLiDownLoader downLoader, ArrayList<Integer> pages, String url, String parent_id,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url = url;
            this.parent_id = parent_id;
            this.speeches_id=speeches_id;
        }
    }


    public YiBinZeroJuLiDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        YiBinZeroJuLiDownLoader loader = new YiBinZeroJuLiDownLoader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

}
